{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Experiment: \n",
    "\n",
    "Compare prunning by Hebbian Learning and Weight Magnitude.\n",
    "\n",
    "#### Motivation.\n",
    "\n",
    "Verify if Hebbian Learning pruning outperforms pruning by Magnitude\n",
    "\n",
    "#### Conclusions:\n",
    "- No pruning leads (0,0) to acc of 0.976\n",
    "- Pruning all connections at every epoch (1,0) leads to acc of 0.964\n",
    "- Best performing model is still no hebbian pruning, and weight pruning set to 0.2 (0.981)\n",
    "- Pruning only by hebbian learning decreases accuracy\n",
    "- Combining hebbian and weight magnitude is not an improvement compared to simple weight magnitude pruning\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append(\"../../\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import os\n",
    "import glob\n",
    "import tabulate\n",
    "import pprint\n",
    "import click\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from ray.tune.commands import *\n",
    "from dynamic_sparse.common.browser import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load and check data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "exps = ['neurips_debug_test6', ]\n",
    "paths = [os.path.expanduser(\"~/nta/results/{}\".format(e)) for e in exps]\n",
    "df = load_many(paths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Experiment Name</th>\n",
       "      <th>train_acc_max</th>\n",
       "      <th>train_acc_max_epoch</th>\n",
       "      <th>train_acc_min</th>\n",
       "      <th>train_acc_min_epoch</th>\n",
       "      <th>train_acc_median</th>\n",
       "      <th>train_acc_last</th>\n",
       "      <th>val_acc_max</th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th>val_acc_min</th>\n",
       "      <th>...</th>\n",
       "      <th>momentum</th>\n",
       "      <th>network</th>\n",
       "      <th>num_classes</th>\n",
       "      <th>on_perc</th>\n",
       "      <th>optim_alg</th>\n",
       "      <th>pruning_early_stop</th>\n",
       "      <th>test_noise</th>\n",
       "      <th>use_kwinners</th>\n",
       "      <th>weight_decay</th>\n",
       "      <th>weight_prune_perc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0_hebbian_prune_perc=None,weight_prune_perc=None</td>\n",
       "      <td>0.988333</td>\n",
       "      <td>28</td>\n",
       "      <td>0.923450</td>\n",
       "      <td>0</td>\n",
       "      <td>0.985358</td>\n",
       "      <td>0.988000</td>\n",
       "      <td>0.9768</td>\n",
       "      <td>29</td>\n",
       "      <td>0.9614</td>\n",
       "      <td>...</td>\n",
       "      <td>0.9</td>\n",
       "      <td>MLPHeb</td>\n",
       "      <td>10</td>\n",
       "      <td>0.2</td>\n",
       "      <td>SGD</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1_hebbian_prune_perc=0.2,weight_prune_perc=None</td>\n",
       "      <td>0.974583</td>\n",
       "      <td>27</td>\n",
       "      <td>0.924417</td>\n",
       "      <td>0</td>\n",
       "      <td>0.970733</td>\n",
       "      <td>0.974483</td>\n",
       "      <td>0.9753</td>\n",
       "      <td>5</td>\n",
       "      <td>0.9609</td>\n",
       "      <td>...</td>\n",
       "      <td>0.9</td>\n",
       "      <td>MLPHeb</td>\n",
       "      <td>10</td>\n",
       "      <td>0.2</td>\n",
       "      <td>SGD</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2_hebbian_prune_perc=0.4,weight_prune_perc=None</td>\n",
       "      <td>0.968250</td>\n",
       "      <td>25</td>\n",
       "      <td>0.926067</td>\n",
       "      <td>0</td>\n",
       "      <td>0.963083</td>\n",
       "      <td>0.967533</td>\n",
       "      <td>0.9710</td>\n",
       "      <td>20</td>\n",
       "      <td>0.9623</td>\n",
       "      <td>...</td>\n",
       "      <td>0.9</td>\n",
       "      <td>MLPHeb</td>\n",
       "      <td>10</td>\n",
       "      <td>0.2</td>\n",
       "      <td>SGD</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3_hebbian_prune_perc=0.6,weight_prune_perc=None</td>\n",
       "      <td>0.957933</td>\n",
       "      <td>23</td>\n",
       "      <td>0.926083</td>\n",
       "      <td>0</td>\n",
       "      <td>0.952508</td>\n",
       "      <td>0.957533</td>\n",
       "      <td>0.9673</td>\n",
       "      <td>23</td>\n",
       "      <td>0.9589</td>\n",
       "      <td>...</td>\n",
       "      <td>0.9</td>\n",
       "      <td>MLPHeb</td>\n",
       "      <td>10</td>\n",
       "      <td>0.2</td>\n",
       "      <td>SGD</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4_hebbian_prune_perc=0.8,weight_prune_perc=None</td>\n",
       "      <td>0.943033</td>\n",
       "      <td>22</td>\n",
       "      <td>0.923467</td>\n",
       "      <td>2</td>\n",
       "      <td>0.936533</td>\n",
       "      <td>0.935983</td>\n",
       "      <td>0.9665</td>\n",
       "      <td>9</td>\n",
       "      <td>0.9514</td>\n",
       "      <td>...</td>\n",
       "      <td>0.9</td>\n",
       "      <td>MLPHeb</td>\n",
       "      <td>10</td>\n",
       "      <td>0.2</td>\n",
       "      <td>SGD</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 42 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Experiment Name  train_acc_max  \\\n",
       "0  0_hebbian_prune_perc=None,weight_prune_perc=None       0.988333   \n",
       "1   1_hebbian_prune_perc=0.2,weight_prune_perc=None       0.974583   \n",
       "2   2_hebbian_prune_perc=0.4,weight_prune_perc=None       0.968250   \n",
       "3   3_hebbian_prune_perc=0.6,weight_prune_perc=None       0.957933   \n",
       "4   4_hebbian_prune_perc=0.8,weight_prune_perc=None       0.943033   \n",
       "\n",
       "   train_acc_max_epoch  train_acc_min  train_acc_min_epoch  train_acc_median  \\\n",
       "0                   28       0.923450                    0          0.985358   \n",
       "1                   27       0.924417                    0          0.970733   \n",
       "2                   25       0.926067                    0          0.963083   \n",
       "3                   23       0.926083                    0          0.952508   \n",
       "4                   22       0.923467                    2          0.936533   \n",
       "\n",
       "   train_acc_last  val_acc_max  val_acc_max_epoch  val_acc_min  ...  momentum  \\\n",
       "0        0.988000       0.9768                 29       0.9614  ...       0.9   \n",
       "1        0.974483       0.9753                  5       0.9609  ...       0.9   \n",
       "2        0.967533       0.9710                 20       0.9623  ...       0.9   \n",
       "3        0.957533       0.9673                 23       0.9589  ...       0.9   \n",
       "4        0.935983       0.9665                  9       0.9514  ...       0.9   \n",
       "\n",
       "   network  num_classes  on_perc optim_alg  pruning_early_stop  test_noise  \\\n",
       "0   MLPHeb           10      0.2       SGD                   0       False   \n",
       "1   MLPHeb           10      0.2       SGD                   0       False   \n",
       "2   MLPHeb           10      0.2       SGD                   0       False   \n",
       "3   MLPHeb           10      0.2       SGD                   0       False   \n",
       "4   MLPHeb           10      0.2       SGD                   0       False   \n",
       "\n",
       "   use_kwinners weight_decay weight_prune_perc  \n",
       "0         False       0.0001               NaN  \n",
       "1         False       0.0001               NaN  \n",
       "2         False       0.0001               NaN  \n",
       "3         False       0.0001               NaN  \n",
       "4         False       0.0001               NaN  \n",
       "\n",
       "[5 rows x 42 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.2])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['on_perc'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# replace hebbian prine\n",
    "df['hebbian_prune_perc'] = df['hebbian_prune_perc'].replace(np.nan, 0.0, regex=True)\n",
    "df['weight_prune_perc'] = df['weight_prune_perc'].replace(np.nan, 0.0, regex=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Experiment Name', 'train_acc_max', 'train_acc_max_epoch',\n",
       "       'train_acc_min', 'train_acc_min_epoch', 'train_acc_median',\n",
       "       'train_acc_last', 'val_acc_max', 'val_acc_max_epoch', 'val_acc_min',\n",
       "       'val_acc_min_epoch', 'val_acc_median', 'val_acc_last', 'epochs',\n",
       "       'experiment_file_name', 'trial_time', 'mean_epoch_time', 'batch_norm',\n",
       "       'data_dir', 'dataset_name', 'debug_sparse', 'debug_weights', 'device',\n",
       "       'hebbian_grow', 'hebbian_prune_perc', 'hidden_sizes', 'input_size',\n",
       "       'learning_rate', 'lr_gamma', 'lr_milestones', 'lr_scheduler', 'model',\n",
       "       'momentum', 'network', 'num_classes', 'on_perc', 'optim_alg',\n",
       "       'pruning_early_stop', 'test_noise', 'use_kwinners', 'weight_decay',\n",
       "       'weight_prune_perc'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(108, 42)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Experiment Name           1_hebbian_prune_perc=0.2,weight_prune_perc=None\n",
       "train_acc_max                                                    0.974583\n",
       "train_acc_max_epoch                                                    27\n",
       "train_acc_min                                                    0.924417\n",
       "train_acc_min_epoch                                                     0\n",
       "train_acc_median                                                 0.970733\n",
       "train_acc_last                                                   0.974483\n",
       "val_acc_max                                                        0.9753\n",
       "val_acc_max_epoch                                                       5\n",
       "val_acc_min                                                        0.9609\n",
       "val_acc_min_epoch                                                       0\n",
       "val_acc_median                                                     0.9694\n",
       "val_acc_last                                                       0.9672\n",
       "epochs                                                                 30\n",
       "experiment_file_name    /Users/lsouza/nta/results/neurips_debug_test6/...\n",
       "trial_time                                                        19.5668\n",
       "mean_epoch_time                                                  0.652226\n",
       "batch_norm                                                           True\n",
       "data_dir                                        /home/ubuntu/nta/datasets\n",
       "dataset_name                                                        MNIST\n",
       "debug_sparse                                                         True\n",
       "debug_weights                                                        True\n",
       "device                                                               cuda\n",
       "hebbian_grow                                                        False\n",
       "hebbian_prune_perc                                                    0.2\n",
       "hidden_sizes                                                          100\n",
       "input_size                                                            784\n",
       "learning_rate                                                         0.1\n",
       "lr_gamma                                                              0.1\n",
       "lr_milestones                                                          60\n",
       "lr_scheduler                                                  MultiStepLR\n",
       "model                                                        DSNNMixedHeb\n",
       "momentum                                                              0.9\n",
       "network                                                            MLPHeb\n",
       "num_classes                                                            10\n",
       "on_perc                                                               0.2\n",
       "optim_alg                                                             SGD\n",
       "pruning_early_stop                                                      0\n",
       "test_noise                                                          False\n",
       "use_kwinners                                                        False\n",
       "weight_decay                                                       0.0001\n",
       "weight_prune_perc                                                       0\n",
       "Name: 1, dtype: object"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.iloc[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "model\n",
       "DSNNMixedHeb    108\n",
       "Name: model, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby('model')['model'].count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " ## Analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Experiment Details"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "base_exp_config = dict(\n",
    "    device=\"cuda\",\n",
    "    # dataset related\n",
    "    dataset_name=\"MNIST\",\n",
    "    data_dir=os.path.expanduser(\"~/nta/datasets\"),\n",
    "    input_size=784,\n",
    "    num_classes=10,\n",
    "    # network related\n",
    "    network=\"MLPHeb\",\n",
    "    hidden_sizes=[100, 100, 100],\n",
    "    batch_norm=True,\n",
    "    use_kwinners=tune.grid_search([True, False]),\n",
    "    # model related\n",
    "    model=\"DSNNMixedHeb\",\n",
    "    on_perc=0.2,\n",
    "    optim_alg=\"SGD\",\n",
    "    momentum=0.9,\n",
    "    weight_decay=1e-4,    \n",
    "    learning_rate=0.1,\n",
    "    lr_scheduler=\"MultiStepLR\",\n",
    "    lr_milestones=[30,60,90],\n",
    "    lr_gamma=0.1,\n",
    "    # sparse related\n",
    "    hebbian_prune_perc=tune.grid_search([0, 0.1, 0.2, 0.3, 0.4, 0.5]),\n",
    "    pruning_early_stop=0,\n",
    "    hebbian_grow=tune.grid_search([True, False]),\n",
    "    # additional validation\n",
    "    test_noise=False,\n",
    "    # debugging\n",
    "    debug_weights=True,\n",
    "    debug_sparse=True,\n",
    "    stop={\"training_iteration\": 30},\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Did any  trials failed?\n",
    "df[df[\"epochs\"]<30][\"epochs\"].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(108, 42)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Removing failed or incomplete trials\n",
    "df_origin = df.copy()\n",
    "df = df_origin[df_origin[\"epochs\"]>=30]\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Series([], Name: epochs, dtype: int64)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# which ones failed?\n",
    "# failed, or still ongoing?\n",
    "df_origin['failed'] = df_origin[\"epochs\"]<30\n",
    "df_origin[df_origin['failed']]['epochs']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# helper functions\n",
    "def mean_and_std(s):\n",
    "    return \"{:.3f} ± {:.3f}\".format(s.mean(), s.std())\n",
    "\n",
    "def round_mean(s):\n",
    "    return \"{:.0f}\".format(round(s.mean()))\n",
    "\n",
    "stats = ['min', 'max', 'mean', 'std']\n",
    "\n",
    "def agg(columns, filter=None, round=3):\n",
    "    if filter is None:\n",
    "        return (df.groupby(columns)\n",
    "             .agg({'val_acc_max_epoch': round_mean,\n",
    "                   'val_acc_max': stats,                \n",
    "                   'model': ['count']})).round(round)\n",
    "    else:\n",
    "        return (df[filter].groupby(columns)\n",
    "             .agg({'val_acc_max_epoch': round_mean,\n",
    "                   'val_acc_max': stats,                \n",
    "                   'model': ['count']})).round(round)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### What are optimal levels of hebbian and weight pruning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ignoring experiments where weight_prune_perc = 1, results not reliable\n",
    "filter = (df['weight_prune_perc'] < 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">val_acc_max</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hebbian_prune_perc</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>23</td>\n",
       "      <td>0.976</td>\n",
       "      <td>0.982</td>\n",
       "      <td>0.979</td>\n",
       "      <td>0.002</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.2</th>\n",
       "      <td>17</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.982</td>\n",
       "      <td>0.978</td>\n",
       "      <td>0.003</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.4</th>\n",
       "      <td>21</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.981</td>\n",
       "      <td>0.977</td>\n",
       "      <td>0.004</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.6</th>\n",
       "      <td>20</td>\n",
       "      <td>0.967</td>\n",
       "      <td>0.982</td>\n",
       "      <td>0.977</td>\n",
       "      <td>0.004</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.8</th>\n",
       "      <td>18</td>\n",
       "      <td>0.964</td>\n",
       "      <td>0.981</td>\n",
       "      <td>0.977</td>\n",
       "      <td>0.006</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.0</th>\n",
       "      <td>19</td>\n",
       "      <td>0.963</td>\n",
       "      <td>0.982</td>\n",
       "      <td>0.977</td>\n",
       "      <td>0.007</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   val_acc_max_epoch val_acc_max                      model\n",
       "                          round_mean         min    max   mean    std count\n",
       "hebbian_prune_perc                                                         \n",
       "0.0                               23       0.976  0.982  0.979  0.002    15\n",
       "0.2                               17       0.973  0.982  0.978  0.003    15\n",
       "0.4                               21       0.970  0.981  0.977  0.004    15\n",
       "0.6                               20       0.967  0.982  0.977  0.004    15\n",
       "0.8                               18       0.964  0.981  0.977  0.006    15\n",
       "1.0                               19       0.963  0.982  0.977  0.007    15"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg(['hebbian_prune_perc'], filter)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* No relevant difference "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">val_acc_max</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weight_prune_perc</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>13</td>\n",
       "      <td>0.963</td>\n",
       "      <td>0.977</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.005</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.2</th>\n",
       "      <td>23</td>\n",
       "      <td>0.978</td>\n",
       "      <td>0.982</td>\n",
       "      <td>0.980</td>\n",
       "      <td>0.001</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.4</th>\n",
       "      <td>22</td>\n",
       "      <td>0.977</td>\n",
       "      <td>0.982</td>\n",
       "      <td>0.980</td>\n",
       "      <td>0.001</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.6</th>\n",
       "      <td>20</td>\n",
       "      <td>0.978</td>\n",
       "      <td>0.981</td>\n",
       "      <td>0.979</td>\n",
       "      <td>0.001</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.8</th>\n",
       "      <td>20</td>\n",
       "      <td>0.976</td>\n",
       "      <td>0.980</td>\n",
       "      <td>0.978</td>\n",
       "      <td>0.001</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  val_acc_max_epoch val_acc_max                      model\n",
       "                         round_mean         min    max   mean    std count\n",
       "weight_prune_perc                                                         \n",
       "0.0                              13       0.963  0.977  0.970  0.005    18\n",
       "0.2                              23       0.978  0.982  0.980  0.001    18\n",
       "0.4                              22       0.977  0.982  0.980  0.001    18\n",
       "0.6                              20       0.978  0.981  0.979  0.001    18\n",
       "0.8                              20       0.976  0.980  0.978  0.001    18"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filter = (df['weight_prune_perc'] < 1)\n",
    "agg(['weight_prune_perc'], filter)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Optimal level between 0.2 and 0.4 (consistent with previous experiments and SET paper, where 0.3 is an optimal value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">val_acc_max</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weight_prune_perc</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>22</td>\n",
       "      <td>0.976</td>\n",
       "      <td>0.977</td>\n",
       "      <td>0.976</td>\n",
       "      <td>0.000</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.2</th>\n",
       "      <td>25</td>\n",
       "      <td>0.979</td>\n",
       "      <td>0.982</td>\n",
       "      <td>0.981</td>\n",
       "      <td>0.001</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.4</th>\n",
       "      <td>24</td>\n",
       "      <td>0.980</td>\n",
       "      <td>0.981</td>\n",
       "      <td>0.980</td>\n",
       "      <td>0.000</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  val_acc_max_epoch val_acc_max                      model\n",
       "                         round_mean         min    max   mean    std count\n",
       "weight_prune_perc                                                         \n",
       "0.0                              22       0.976  0.977  0.976  0.000     3\n",
       "0.2                              25       0.979  0.982  0.981  0.001     3\n",
       "0.4                              24       0.980  0.981  0.980  0.000     3"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "magonly = (df['hebbian_prune_perc'] == 0.0) & (df['weight_prune_perc'] < 0.6) \n",
    "agg(['weight_prune_perc'], magonly)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### What is the optimal combination of both"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>weight_prune_perc</th>\n",
       "      <th>0.0</th>\n",
       "      <th>0.2</th>\n",
       "      <th>0.4</th>\n",
       "      <th>0.6</th>\n",
       "      <th>0.8</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hebbian_prune_perc</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>0.976 ± 0.000</td>\n",
       "      <td>0.981 ± 0.001</td>\n",
       "      <td>0.980 ± 0.000</td>\n",
       "      <td>0.980 ± 0.001</td>\n",
       "      <td>0.979 ± 0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.2</th>\n",
       "      <td>0.974 ± 0.001</td>\n",
       "      <td>0.979 ± 0.001</td>\n",
       "      <td>0.979 ± 0.003</td>\n",
       "      <td>0.979 ± 0.002</td>\n",
       "      <td>0.977 ± 0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.4</th>\n",
       "      <td>0.971 ± 0.001</td>\n",
       "      <td>0.980 ± 0.001</td>\n",
       "      <td>0.979 ± 0.001</td>\n",
       "      <td>0.979 ± 0.001</td>\n",
       "      <td>0.978 ± 0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.6</th>\n",
       "      <td>0.969 ± 0.001</td>\n",
       "      <td>0.979 ± 0.001</td>\n",
       "      <td>0.980 ± 0.001</td>\n",
       "      <td>0.979 ± 0.001</td>\n",
       "      <td>0.978 ± 0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.8</th>\n",
       "      <td>0.966 ± 0.001</td>\n",
       "      <td>0.980 ± 0.001</td>\n",
       "      <td>0.980 ± 0.002</td>\n",
       "      <td>0.980 ± 0.001</td>\n",
       "      <td>0.978 ± 0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.0</th>\n",
       "      <td>0.964 ± 0.001</td>\n",
       "      <td>0.981 ± 0.001</td>\n",
       "      <td>0.981 ± 0.001</td>\n",
       "      <td>0.980 ± 0.001</td>\n",
       "      <td>0.980 ± 0.001</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "weight_prune_perc             0.0            0.2            0.4  \\\n",
       "hebbian_prune_perc                                                \n",
       "0.0                 0.976 ± 0.000  0.981 ± 0.001  0.980 ± 0.000   \n",
       "0.2                 0.974 ± 0.001  0.979 ± 0.001  0.979 ± 0.003   \n",
       "0.4                 0.971 ± 0.001  0.980 ± 0.001  0.979 ± 0.001   \n",
       "0.6                 0.969 ± 0.001  0.979 ± 0.001  0.980 ± 0.001   \n",
       "0.8                 0.966 ± 0.001  0.980 ± 0.001  0.980 ± 0.002   \n",
       "1.0                 0.964 ± 0.001  0.981 ± 0.001  0.981 ± 0.001   \n",
       "\n",
       "weight_prune_perc             0.6            0.8  \n",
       "hebbian_prune_perc                                \n",
       "0.0                 0.980 ± 0.001  0.979 ± 0.001  \n",
       "0.2                 0.979 ± 0.002  0.977 ± 0.001  \n",
       "0.4                 0.979 ± 0.001  0.978 ± 0.001  \n",
       "0.6                 0.979 ± 0.001  0.978 ± 0.001  \n",
       "0.8                 0.980 ± 0.001  0.978 ± 0.000  \n",
       "1.0                 0.980 ± 0.001  0.980 ± 0.001  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.pivot_table(df[filter], \n",
    "              index='hebbian_prune_perc',\n",
    "              columns='weight_prune_perc',\n",
    "              values='val_acc_max',\n",
    "              aggfunc=mean_and_std)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>weight_prune_perc</th>\n",
       "      <th>0.0</th>\n",
       "      <th>0.2</th>\n",
       "      <th>0.4</th>\n",
       "      <th>0.6</th>\n",
       "      <th>0.8</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hebbian_prune_perc</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>0.974 ± 0.003</td>\n",
       "      <td>0.980 ± 0.001</td>\n",
       "      <td>0.977 ± 0.002</td>\n",
       "      <td>0.978 ± 0.001</td>\n",
       "      <td>0.976 ± 0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.2</th>\n",
       "      <td>0.969 ± 0.002</td>\n",
       "      <td>0.978 ± 0.002</td>\n",
       "      <td>0.976 ± 0.001</td>\n",
       "      <td>0.977 ± 0.001</td>\n",
       "      <td>0.975 ± 0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.4</th>\n",
       "      <td>0.967 ± 0.000</td>\n",
       "      <td>0.977 ± 0.002</td>\n",
       "      <td>0.978 ± 0.002</td>\n",
       "      <td>0.975 ± 0.002</td>\n",
       "      <td>0.977 ± 0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.6</th>\n",
       "      <td>0.967 ± 0.003</td>\n",
       "      <td>0.977 ± 0.001</td>\n",
       "      <td>0.978 ± 0.001</td>\n",
       "      <td>0.976 ± 0.002</td>\n",
       "      <td>0.975 ± 0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.8</th>\n",
       "      <td>0.961 ± 0.000</td>\n",
       "      <td>0.979 ± 0.001</td>\n",
       "      <td>0.978 ± 0.001</td>\n",
       "      <td>0.978 ± 0.002</td>\n",
       "      <td>0.976 ± 0.002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.0</th>\n",
       "      <td>0.954 ± 0.001</td>\n",
       "      <td>0.979 ± 0.000</td>\n",
       "      <td>0.979 ± 0.001</td>\n",
       "      <td>0.978 ± 0.001</td>\n",
       "      <td>0.976 ± 0.002</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "weight_prune_perc             0.0            0.2            0.4  \\\n",
       "hebbian_prune_perc                                                \n",
       "0.0                 0.974 ± 0.003  0.980 ± 0.001  0.977 ± 0.002   \n",
       "0.2                 0.969 ± 0.002  0.978 ± 0.002  0.976 ± 0.001   \n",
       "0.4                 0.967 ± 0.000  0.977 ± 0.002  0.978 ± 0.002   \n",
       "0.6                 0.967 ± 0.003  0.977 ± 0.001  0.978 ± 0.001   \n",
       "0.8                 0.961 ± 0.000  0.979 ± 0.001  0.978 ± 0.001   \n",
       "1.0                 0.954 ± 0.001  0.979 ± 0.000  0.979 ± 0.001   \n",
       "\n",
       "weight_prune_perc             0.6            0.8  \n",
       "hebbian_prune_perc                                \n",
       "0.0                 0.978 ± 0.001  0.976 ± 0.001  \n",
       "0.2                 0.977 ± 0.001  0.975 ± 0.001  \n",
       "0.4                 0.975 ± 0.002  0.977 ± 0.003  \n",
       "0.6                 0.976 ± 0.002  0.975 ± 0.001  \n",
       "0.8                 0.978 ± 0.002  0.976 ± 0.002  \n",
       "1.0                 0.978 ± 0.001  0.976 ± 0.002  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.pivot_table(df[filter], \n",
    "              index='hebbian_prune_perc',\n",
    "              columns='weight_prune_perc',\n",
    "              values='val_acc_last',\n",
    "              aggfunc=mean_and_std)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(108, 42)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Conclusions:\n",
    "- No pruning leads (0,0) to acc of 0.976\n",
    "- Pruning all connections at every epoch (1,0) leads to acc of 0.964\n",
    "- Best performing model is still no hebbian pruning, and weight pruning set to 0.2 (0.981)\n",
    "- Pruning only by hebbian learning decreases accuracy\n",
    "- Combining hebbian and weight magnitude is not an improvement compared to simple weight magnitude pruning\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
